<!DOCTYPE html>
<html lang="en">
  <head>
    <!-- AdThrive Head Tag Manual -->
    <script data-no-optimize="1" data-cfasync="false">
      (function (w, d) {
        w.adthrive = w.adthrive || {};
        w.adthrive.cmd = w.adthrive.cmd || [];
        w.adthrive.plugin = "adthrive-ads-manual";
        w.adthrive.host = "ads.adthrive.com";
        var s = d.createElement("script");
        s.async = true;
        s.referrerpolicy = "no-referrer-when-downgrade";
        s.src =
          "https://" +
          w.adthrive.host +
          "/sites/643436a4e6d20859e40a446b/ads.min.js?referrer=" +
          w.encodeURIComponent(w.location.href) +
          "&cb=" +
          (Math.floor(Math.random() * 100) + 1);
        var n = d.getElementsByTagName("script")[0];
        n.parentNode.insertBefore(s, n);
      })(window, document);
    </script>
    <!-- End of AdThrive Head Tag -->

    <!-- Global site tag (gtag.js) - Google Analytics -->

    <script
      async
      src="https://www.googletagmanager.com/gtag/js?id=UA-79254642-6"
    ></script>

    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag() {
        dataLayer.push(arguments);
      }
      gtag("js", new Date());

      gtag("config", "UA-79254642-6");
    </script>

    <meta charset="utf-8" />
    <title>Data manipulation in d3.js</title>
    <meta
      name="viewport"
      content="width=device-width, initial-scale=1, shrink-to-fit=no"
    />
    <meta
      name="description"
      content="A few code chuncks describing the most common data manipulation needed in d3.js. Includes filtering, sorting, nesting and more."
    />
    <meta
      name="keywords"
      content="Data,Dataviz,Datavisualization,Javascript, JS, d3.js, axis, x, y"
    />
    <meta name="author" content="Yan Holtz" />
    <link rel="icon" href="../img/logo/D3_single_small.png" />

    <meta property="og:title" content="Data manipulation in d3.js" />
    <meta property="og:image" content="img/overview_RGG.png" />
    <meta
      property="og:description"
      content="A few code chuncks describing the most common data manipulation needed in d3.js. Includes filtering, sorting, nesting and more.."
    />

    <!-- Bootstrap core CSS -->
    <link href="../vendor/bootstrap/css/bootstrap.min.css" rel="stylesheet" />

    <!-- Custom fonts for this template -->
    <link
      href="../vendor/font-awesome/css/font-awesome.min.css"
      rel="stylesheet"
      type="text/css"
    />

    <!-- Custom styles for this template -->
    <link href="../css/agency.css" rel="stylesheet" />

    <!-- PRISM -->
    <script src="../LIB/prism.js"></script>
    <link href="../LIB/prism.css" rel="stylesheet" />

    <!-- D3.JS v4 -->
    <script src="../js/d3.v4.js"></script>

    <!-- JQUERY -->
    <script src="../vendor/jquery/jquery.min.js"></script>

    <!-- HTML TO CANVAS -->
    <script src="../js/html2canvas.js"></script>

    <!-- Function to parse html and js code chunks made by Yan Holtz -->
    <script src="../js/myParser.js"></script>

    <!-- Bootstrap js -->
    <script src="../vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
    <script src="../js/agency.min.js"></script>
  </head>

  <body data-spy="scroll" data-target="#myScrollspy" data-offset="1">
    <!-- THIS ALLOWS TO INSERT THE MENU THAT IS STORED IN A MENU.HTML FILE-->
    <nav class="navbar navbar-expand-lg fixed-top" id="mainNav"></nav>
    <script>
      $(function () {
        $("#mainNav").load("../html_chunk/menu.html");
      });
    </script>

    <!-- THIS ALLOWS TO INSERT THE MODAL OF THE MENU THAT IS STORED IN A MENU_MODAL.HTML FILE-->
    <div id="modal_menu_insertion"></div>
    <script>
      $(function () {
        $("#modal_menu_insertion").load("../html_chunk/menu_modal.html");
      });
    </script>

    <!-- Header -->
    <header class="masthead" style="padding-top: 150px; padding-bottom: 80px">
      <div class="textlanding">
        <h1>Data manipulation in d3.js</h1>
        <hr class="short_hr" />
        <br />
        <ul class="list-inline social-buttons">
          <li class="list-inline-item">
            <a href="https://twitter.com/R_Graph_Gallery">
              <i class="fa fa-twitter"></i>
            </a>
          </li>
          <li class="list-inline-item social-buttons">
            <a href="https://github.com/holtzy">
              <i class="fa fa-github" style="color: white"></i>
            </a>
          </li>
          <li class="list-inline-item social-buttons">
            <a href="https://www.linkedin.com/in/yan-holtz-2477534a/">
              <i class="fa fa-linkedin"></i>
            </a>
          </li>
        </ul>
        <br /><br />
        <p style="max-width: 700px; margin: auto">
          This post describes the most common <u>data manipulation tasks</u> you
          will have to perform using <code>d3.js</code>. It includes
          <a href="#sorting">sorting</a>, <a href="#filtering">filtering</a>,
          <a href="#grouping">grouping</a>, <a href="#nesting">nesting</a> and
          more.
        </p>
      </div>
    </header>

    <!-- TABLE of CONTENT -->
    <div>
      <nav class="col-sm-3 col-4" id="myScrollspy">
        <ul class="nav nav-pills flex-column">
          <li class="nav-item">
            <a class="nav-link" href="#math">Mathematics</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#list">Lists and Arrays</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#filtering">Filter</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#sorting">Sort</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#nesting">Nest</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#grouping">Group</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#loop">Loop</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#reshape">Reshape</a>
          </li>
          <li class="nav-item">
            <a class="nav-link" href="#stack">Stack</a>
          </li>
        </ul>
      </nav>
    </div>

    <!-- ==================== MATH ==================== -->

    <section id="math">
      <div class="container">
        <h1>Mathematics</h1>
        <hr />

        <h6>Get the maximum value of a column called <code>value</code></h6>
        <pre class="language-js">
d3.max(data, function(d) { return +d.value; })])
</pre
        >
        <br />

        <h6>Get the minimum value of a column called <code>value</code></h6>
        <pre class="language-js">
d3.min(data, function(d) { return +d.value; })])
</pre
        >
        <br />
      </div>
    </section>

    <!-- ==================== MATH ==================== -->

    <section id="list">
      <div class="container">
        <h1>Object and Arrays</h1>
        <hr />

        <h6>
          JavaScript <code>objects</code> are containers for named values called
          properties or methods. We can call any element of the object with its
          <u>name</u>:
        </h6>
        <pre class="language-js">
var myObject = {name:"Nicolas", sex:"Male", age:34}
myObject.name
myObject["name"]
</pre
        >
        <br />

        <h6>Create an <code>array</code> of 5 numbers</h6>
        <p>
          An array is a special variable, which can hold more than one value at
          a time. Arrays use numbered indexes.
        </p>
        <pre class="language-js">
var myArray = [12, 34, 23, 12, 89]
</pre
        >
        <br />

        <h6>Access the first element in the array.</h6>
        <pre class="language-js">
myArray[0]
</pre
        >
        <br />

        <h6>Access the last element in the array.</h6>
        <pre class="language-js">
myArray[myArray.length - 1]
</pre
        >
        <br />

        <h6>Remove last element of the array. Add a new element a the end.</h6>
        <pre class="language-js">
myArray.pop()
myArray.push(45)
</pre
        >
        <br />

        <h6>What is the index of the element '34' in the array?.</h6>
        <pre class="language-js">
myArray.indexOf(34)
</pre
        >
        <br />

        <h6>Remove elements that are provided in a second array:</h6>
        <pre class="language-js">
var myArray = ['a', 'b', 'c', 'd', 'e', 'f', 'g'];
var toRemove = ['b', 'c', 'g'];
filteredArray = myArray.filter( function( el ) {
  return !toRemove.includes( el );
} );
</pre
        >
        <br />

        <h6>Keep only elements that are provided in a second array:</h6>
        <pre class="language-js">
var myArray = ['a', 'b', 'c', 'd', 'e', 'f', 'g'];
var tokeep = ['b', 'c', 'g', 'o', 'z'];
filteredArray = myArray.filter( function( el ) {
  return tokeep.includes( el );
} );
</pre
        >
        <br />

        <u>Note</u>: an array can be composed by several objects!
      </div>
    </section>

    <!-- ==================== FILTERING ==================== -->

    <section id="filtering">
      <div class="container">
        <h1>Filtering</h1>
        <hr />

        <h6>
          Keep rows where the variable <code>name</code> is <code>toto</code>
        </h6>
        <pre class="language-js">
data.filter(function(d){ return d.name == "toto" })
</pre
        >
        <br />

        <h6>
          Keep rows where the variable <code>name</code> is
          <u>different</u> from <code>toto</code>
        </h6>
        <pre class="language-js">
data.filter(function(d){ return d.name != "toto" })
</pre
        >
        <br />

        <h6>
          Keep rows where the variable <code>name</code> is <code>toto</code> OR
          <code>tutu</code>
        </h6>
        <pre class="language-js">
data.filter(function(d){ return  (d.name == "toto" || d.name == "tutu") })
</pre
        >
        <br />

        <h6>
          Keep rows where the variable <code>name</code> has a value included in
          the list <code>tokeep</code>
        </h6>
        <pre class="language-js">
tokeep = ["option1", "option2", "option3"]
data.filter(function(d,i){ return tokeep.indexOf(d.name) >= 0 })
</pre
        >
        <br />

        <h6>Keep the 10 first rows</h6>
        <pre class="language-js">
data.filter(function(d,i){ return i<10 })
</pre
        >
        <br />

        <h6>color points using <code>ifelse</code> statement</h6>
        <pre class="language-js">
.style("fill", function(d){ if(d.x<140){return "orange"} else {return "blue"}})
</pre
        >
        <br />
      </div>
    </section>

    <!-- ==================== SORTING==================== -->

    <section id="sorting">
      <div class="container">
        <h1>Sorting</h1>
        <hr />

        <h6>
          Sorting on 1 numeric column called <code>value</code>. Use
          <code>+</code> instead of <code>-</code> for reverse order.
        </h6>
        <pre class="language-js">
data.sort(function(a,b) { return +a.value - +b.value })
</pre
        >
        <br />

        <h6>
          Sorting alphabetically on 1 categoric column called <code>name</code>.
          Use <code>descending</code> for reverse order.
        </h6>
        <pre class="language-js">
data.sort(function (a,b) {return d3.ascending(a.name, b.name);});
</pre
        >
        <br />

        <h6>
          Sorting alphabetically on 2 categoric columns called
          <code>name1</code> and <code>name2</code>.
        </h6>
        <pre class="language-js">
data.sort(function(a,b) { return d3.ascending(a.name1, b.name1) ||  d3.ascending(a.name1, b.name2) } )
</pre
        >
        <br />

        <h6>
          Sorting on 1 categoric columns called <code>name1</code> and then on 1
          numeric called <code>value</code>.
        </h6>
        <pre class="language-js">
data.sort(function(a,b) { return d3.ascending(a.name1, b.name1) ||  (a.value - b.value) } )
</pre
        >
        <br />

        <h6>
          Sorting on 1 categoric columns called <code>name1</code> according to
          the order provided in the variable <code>targetOrder</code>.
        </h6>
        <pre class="language-js">
data.sort(function(a,b) {
    return targetOrder.indexOf( a.name1 ) > targetOrder.indexOf( b.name1 );
});
</pre
        >
        <br />
      </div>
    </section>

    <!-- ==================== NESTING ==================== -->

    <section id="nesting">
      <div class="container">
        <h1>Nesting</h1>
        <hr />

        <h6>
          Sorting on 1 numeric column called <code>value</code>. Use
          <code>+</code> instead of <code>-</code> for reverse order.
        </h6>
        <pre class="language-js">
data.sort(function(a,b) { return +a.value - +b.value })
</pre
        >
        <br />

        <h6>
          Sorting alphabetically on 1 categoric column called <code>name</code>.
          Use <code>descending</code> for reverse order.
        </h6>
        <pre class="language-js">
data.sort(function (a,b) {return d3.ascending(a.name, b.name);});
</pre
        >
        <br />
      </div>
    </section>

    <!-- ==================== GROUPING ==================== -->

    <section id="grouping">
      <div class="container">
        <h1>Grouping</h1>
        <hr />

        <h6>
          Get a list of unique entries of a column called <code>name</code>
        </h6>
        <pre class="language-js">
var allGroup = d3.map(data, function(d){return(d.name)}).keys()
</pre
        >
        <br />
      </div>
    </section>

    <!-- ==================== GROUPING ==================== -->

    <section id="loop">
      <div class="container">
        <h1>Loop</h1>
        <hr />

        <h6>A <code>for</code> loop from one to ten:</h6>
        <pre class="language-js">
var i
for (i = 0; i < 10; i++) {
  console.log(i)
}
</pre
        >
        <br />

        <h6>
          A <code>for</code> loop for all the elements of a list: (Note that it
          returns 0, 1, 2, not a, b, c)
        </h6>
        <pre class="language-js">
var allGroup = ["a", "b", "c"]
for (i in allGroup){
  console.log(i)
}
</pre
        >
        <br />

        <h6>A <code>while</code> loop to count from 0 to 10</h6>
        <pre class="language-js">
while (i < 10) {
  console.log(i)
  i++;
}
</pre
        >
        <br />
      </div>
    </section>

    <!-- ==================== RESHAPE ==================== -->

    <section id="reshape">
      <div class="container">
        <h1>Reshape</h1>
        <hr />
        <p>
          It is a common task in data science to swap between <u>wide</u> (or
          untidy) format to <u>long</u> (or tidy) format. In R, there is a
          package called <code>tidyr</code> that is entirely dedicated to it. It
          is definitely doable in Javascript using the code snippets below. In
          case you're not familiar with this concept, here is a description of
          what these formats are:
        </p>
        <center>
          <img
            width="650px"
            src="../img/other/long_vs_wide_data_format.png"
            alt="long versus wide data format summary"
            style="
              padding: 30px;
              margin: 20px;
              border: solid;
              border-width: 1px;
              border-color: #b8b8b8;
              border-radius: 4px;
            "
          />
        </center>
        <p>
          <u>Note</u>: it is strongly advised to perform these data wrangling
          steps out of your javascript to save <u>loading time</u> of your
          dataviz
        </p>
        <br />

        <h6>Going from <code>wide to long</code> format.</h6>
        <pre class="language-js">
d3.csv("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/data_correlogram.csv", function(data) {

  // Going from wide to long format
  var data_long = [];
  data.forEach(function(d) {
    var x = d[""];
    delete d[""];
    for (prop in d) {
      var y = prop,
        value = d[prop];
      data_long.push({
        x: x,
        y: y,
        value: +value
      });
    }
  });

  // Show result
  console.log(data_long)
</pre
        >
        <br />

        <br />
        <h6>Going from <code>long to wide</code> format.</h6>
        <pre class="language-js">
d3.csv("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/data.csv", function(data) {
  //todo
})
</pre
        >
      </div>
    </section>

    <!-- ==================== RESHAPE ==================== -->

    <section id="stack">
      <div class="container">
        <h1>Stack</h1>
        <hr />
        <p>
          Stacking data is a common practice in dataviz, notably for
          <a href="../barplot.html">barcharts</a> and
          <a href="../area.html">area charts</a>. Stacking applies when a
          dataset is composed by <u>groups</u> (like species) and
          <u>subgroups</u> like soil condition. Stacking is possible thanks to
          the <code>d3.stack()</code> function, which is part of the
          <a href="https://github.com/d3/d3-shape#stacks">d3-shape module</a>.
          Here is an illustration of what happens when you read data from
          <code>.csv</code> format and stack it.
        </p>
        <center>
          <img
            width="950px"
            src="../img/other/stackingExpla.png"
            alt="Stacking data explanation"
            style="
              padding: 30px;
              margin: 20px;
              border: solid;
              border-width: 1px;
              border-color: #b8b8b8;
              border-radius: 4px;
            "
          />
        </center>
        <br />

        <script>
          d3.csv(
            "https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/data_stacked.csv",
            function (data) {
              console.log(data);
              // List of subgroups = header of the csv files = soil condition here
              var subgroups = data.columns.slice(1);

              // List of groups = species here = value of the first column called group
              var groups = d3
                .map(data, function (d) {
                  return d.group;
                })
                .keys();

              //stack the data? --> stack per subgroup
              var stackedData = d3.stack().keys(subgroups)(data);

              console.log(stackedData);
            }
          );
        </script>

        <h6>Stacking from <code>.csv</code> format</h6>
        <pre class="language-js">
d3.csv("https://raw.githubusercontent.com/holtzy/D3-graph-gallery/master/DATA/data_stacked.csv", function(data) {

  // Have a look to the data
  console.log(data)

  // List of subgroups = header of the csv files = soil condition here
  var subgroups = data.columns.slice(1)

  // List of groups = species here = value of the first column called group
  var groups = d3.map(data, function(d){return(d.group)}).keys()

  //stack the data? --> stack per subgroup
  var stackedData = d3.stack()
    .keys(subgroups)
    (data)

  // Have a look to the stacked data
  console.log(stackedData)

  // Get the stacked data for the second group

})
</pre
        >
        <br />

        <h6>Get the key of each element of stackedData ???</h6>
      </div>
    </section>

    <!-- ============================ CONTACT SECTION ============================ -->

    <!-- ANCHOR -->
    <a name="contactanchor"></a>

    <section id="contact" class="bg" style="background-color: white"></section>

    <!-- THIS ALLOWS TO INSERT THE CONTACT CHUNK THAT IS STORED IN A CONTACT.HTML FILE-->
    <script>
      $(function () {
        $("#contact").load("../html_chunk/contact.html");
      });
    </script>

    <!-- ============================ FOOTER SECTION ============================ -->
    <footer class="bg-light" id="myFooter"></footer>

    <!-- Start Added by AdThrive -->
    <script
      type="text/javascript"
      async
      src="https://btloader.com/tag?o=5698917485248512&upapi=true&domain=d3-graph-gallery.com"
    ></script>
    <script>
      !(function () {
        "use strict";
        var e;
        (e = document),
          (function () {
            var t, n;
            function r() {
              var t = e.createElement("script");
              (t.src =
                "https://cafemedia-com.videoplayerhub.com/galleryplayer.js"),
                e.head.appendChild(t);
            }
            function a() {
              var t = e.cookie.match("(^|[^;]+)\s*__adblocker\s*=\s*([^;]+)");
              return t && t.pop();
            }
            function c() {
              clearInterval(n);
            }
            return {
              init: function () {
                var e;
                "true" === (t = a())
                  ? r()
                  : ((e = 0),
                    (n = setInterval(function () {
                      (100 !== e && "false" !== t) || c(),
                        "true" === t && (r(), c()),
                        (t = a()),
                        e++;
                    }, 50)));
              },
            };
          })().init();
      })();
    </script>
    <!-- End Added by AdThrive -->

    <!-- THIS ALLOWS TO INSERT THE FOOTER THAT IS STORED IN A FOOTER.HTML FILE-->
    <script>
      $(function () {
        $("#myFooter").load("../html_chunk/footer.html");
      });
    </script>

    <!-- ============================ -->
  </body>
</html>
